In [104]:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
sns.set()
%matplotlib notebook
In [2]:
colors = ['#e6194b', '#3cb44b', '#ffe119', '#4363d8', '#f58231', '#911eb4',
'#46f0f0', '#f032e6', '#bcf60c', '#fabebe', '#008080', '#e6beff',
'#9a6324', '#fffac8', '#800000', '#aaffc3', '#808000', '#ffd8b1',
'#000075', '#808080']*10
Exploring Data¶
In [3]:
df = pd.read_csv('taxi_data.csv')
df.head()
Out[3]:
In [8]:
print('Count of Dataframe Elements : {}'.format(df.count()))
print('Columns in Dataframe : ', df.columns)
print('Duplicate Values (Longitude and Latitude): {}'.format(df.duplicated(subset = ['LON', 'LAT']).any()))
print('Null Values : ',df.isna().any())
In [11]:
# droping Null values and duplicates
print('Original shape of Dataframev : {}'.format(df.shape))
df = df.drop_duplicates(subset=['LON','LAT']).dropna()
print('New Shape after Filtering : {}'.format(df.shape))
In [20]:
# Plot b/w Lon and Lat
plt.figure()
X = np.array(df[['LON','LAT']], dtype='float64')
plt.scatter(X[:,0],X[:,1],s=50,alpha=0.2)
plt.xlabel('Longitudes')
plt.ylabel('Latitudes')
plt.title('LAT vs LON')
plt.show();
Visualize Geographical Data¶
Plotting map with lats and lons¶
In [25]:
import folium
import regex as re
m = folium.Map(location=[X[:,1].mean(),X[:,0].mean()],zoom_start=9,tiles = 'Stamen Toner')
for _,row in df.iterrows():
folium.CircleMarker(
location = [row.LAT,row.LON],
radius = 4,
popup = re.sub(r'[^a-zA-Z ]+',' ',row.NAME),
color = '#178FFE',
fill=True,
fill_colour = '#178FFE'
).add_to(m)
In [26]:
m
Out[26]:
Clustering¶
In [27]:
# 1. KMeans Cluster
from sklearn.cluster import KMeans
X = np.array(df[['LON','LAT']], dtype='float64')
k = 70
model = KMeans(n_clusters=k,random_state=17).fit(X)
model_prediction = model.predict(X)
df[f'ClusterMeans{k}'] = model_prediction
In [28]:
df.head()
Out[28]:
In [31]:
def create_map(df, col_name):
m = folium.Map(location=[df.LAT.mean(), df.LON.mean()], zoom_start=9, tiles='Stamen Toner')
for _,row in df.iterrows():
if row[col_name] == -1:
cluster_colour = '#000000'
else:
cluster_colour = colors[row[col_name]]
folium.CircleMarker(
location = [row.LAT,row.LON],
radius = 4,
popup = row[col_name],
color = cluster_colour,
fill = True,
fill_color = cluster_colour
).add_to(m)
return m
In [34]:
m_KMeans = create_map(df, 'ClusterMeans70')
m_KMeans
Out[34]:
In [35]:
# silhouette score
from sklearn.metrics import silhouette_score
In [38]:
print('Silhouette Score for k = 70 : ', silhouette_score(X,model_prediction))
In [41]:
# Best Silhouette Score
from tqdm import tqdm
for k in tqdm(range(2,100)):
model = KMeans(n_clusters=k, random_state =0).fit(X)
model_prediction = model.predict(X)
k_score = silhouette_score(X,model_prediction)
best_score = -1
if k_score > best_score:
K = k
best_score = k_score
print('Best Score for KMeans is for k = {} of {}'.format(K, best_score))
In [43]:
# 2. DBSCAN - Density-Based Spatial Clustering of Applications with Noise
from sklearn.cluster import DBSCAN
model = DBSCAN(eps=0.01, min_samples=5).fit(X)
model_prediction = model.labels_
df['Cluster_DBSCAN'] = model_prediction
df.head()
Out[43]:
In [44]:
m_dbscan = create_map(df, 'Cluster_DBSCAN')
m_dbscan
Out[44]:
In [49]:
# Visualization
print('No. of Clusters Found : ',len(np.unique(model_prediction)))
print('No. of Outliers Found : ',len(model_prediction[model_prediction == -1]))
print('Silhouette Ignoring Outliers : ',silhouette_score(X[model_prediction != -1], model_prediction[model_prediction != -1]))
#each outlier as Singleton
no_outliers = 0
no_outliers = np.array([(counter+2)*x if x == -1 else x for counter, x in enumerate(model_prediction)])
print('Silhouette Score (Outliers as Singletons) : ',silhouette_score(X,no_outliers))
In [53]:
# 3. HDBSCAN - Hierarchical DBSCAN
import hdbscan
model = hdbscan.HDBSCAN(min_cluster_size=5,min_samples=2,cluster_selection_epsilon=0.02)
model_prediction = model.fit_predict(X)
df['Cluster_hdbscan'] = model_prediction
In [54]:
df.head()
Out[54]:
In [56]:
m_hd = create_map(df,'Cluster_hdbscan')
m_hd #Less no of Black Circles.. ie Noise Reduced
Out[56]:
In [60]:
print(f'Number of clusters found: {len(np.unique(model_prediction))-1}')
print(f'Number of outliers found: {len(model_prediction[model_prediction==-1])}')
print(f'Silhouette ignoring outliers: {silhouette_score(X[model_prediction!=-1], model_prediction[model_prediction!=-1])}')
no_outliers = np.array([(counter+2)*x if x==-1 else x for counter, x in enumerate(model_prediction)])
print(f'Silhouette outliers as singletons: {silhouette_score(X, no_outliers)}')
In [63]:
# 4. Agglomerative Clustering
from sklearn.cluster import AgglomerativeClustering
model = AgglomerativeClustering(n_clusters = 3)
model_prediction = model.fit_predict(X)
print('Silhouette Score = ', silhouette_score(X,model_prediction))
In [70]:
# checking best value for n_clusters and linkage
clusters = np.arange(2,100)
linkages = ['ward','average','complete']
for link in linkages:
for n in tqdm(clusters):
best_score = -1
score = silhouette_score(X,AgglomerativeClustering(n_clusters=n, linkage=link).fit_predict(X))
if score > best_score:
linkage = link
n_ = n
best_score = -1
print('Best Linkage is {} with {} clusters with a score of {}'.format(linkage, n_, score))
In [71]:
model_prediction = AgglomerativeClustering(n_clusters=90,linkage='complete').fit_predict(X)
df['Clusters_AC'] = model_prediction
m_AC = create_map(df, 'Clusters_AC')
m_AC
Out[71]:
Dealing With Outliers¶
In [72]:
from sklearn.neighbors import KNeighborsClassifier
In [81]:
classifier = KNeighborsClassifier(n_neighbors=1)
df_train = df[df.Cluster_hdbscan != -1]
df_predict = df[df.Cluster_hdbscan == -1]
In [83]:
X_train = np.array(df_train[['LON','LAT']], dtype='float64')
y_train = np.array(df_train['Cluster_hdbscan'])
X_predict = np.array(df_predict[['LON','LAT']],dtype='float64')
In [84]:
classifier.fit(X_train, y_train)
Out[84]:
In [85]:
predicts = classifier.predict(X_predict)
In [86]:
df['Cluster_hybrid'] = df['Cluster_hdbscan']
In [89]:
df.loc[df.Cluster_hdbscan == -1, 'Cluster_hybrid'] = predicts
In [90]:
m_hybrid = create_map(df,'Cluster_hybrid')
In [91]:
m_hybrid
Out[91]:
In [93]:
class_predictions = df.Cluster_hybrid
print(f'Number of clusters found: {len(np.unique(class_predictions))}')
print(f'Silhouette: {silhouette_score(X, class_predictions)}')
m.save('hybrid.html')
In [94]:
# Silhouette Score has decreased.. but now the searching clusters based on density which is good in our case
In [115]:
plt.figure()
df['Cluster_hybrid'].value_counts().plot.hist(bins=70, alpha=0.7, label='Hybrid')
df['ClusterMeans70'].value_counts().plot.hist(bins=70, alpha=0.5,label='K-Means')
plt.xlabel('Cluster Size')
plt.legend()
plt.show();
In [ ]: